#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sh
import subprocess

def main():
#     print sh.ls('-l')



#     mk input dir 
#     mkdir1 = '~/hadoop-2.6.0/bin/hdfs dfs -mkdir /Social'
#     ~/hadoop-2.6.0/bin/hdfs dfs -mkdir /Social/input
#     subprocess.call(mkdir1, shell=True)

    
    #for every file in ~/git/Social/Social/resource/daybyday
    for date_num in range(20151031, 20151032):
#         date_str = str(date_num)
         
        #delete local output
        delete_local = 'rm -rf ~/git/Social/Social/resource/hot_topic_user_count/output'
        subprocess.call(delete_local, shell=True)
         
        #delete output
        delete_output = '~/hadoop-2.6.0/bin/hdfs dfs -rm -r /Social/output'
        delete_input = '~/hadoop-2.6.0/bin/hdfs dfs -rm -r /Social/input'
        subprocess.call(delete_output, shell=True)
        subprocess.call(delete_input, shell=True)
         
        #mk input dir 
        mkdir = '~/hadoop-2.6.0/bin/hdfs dfs -mkdir /Social/input'
        subprocess.call(mkdir, shell=True)
        #put input file
        put_file = '~/hadoop-2.6.0/bin/hdfs dfs -put '\
                    '~/git/Social/Social/resource/daybyday/%d.txt ' \
                    '/Social/input' % date_num
        subprocess.call(put_file, shell=True)
        #run
        run_hadoop = '''~/hadoop-2.6.0/bin/hadoop jar \
                        ~/hadoop-2.6.0/share/hadoop/tools/lib/hadoop-streaming*.jar \
                        -file ~/git/Social/Social/src/hadoop/hot_topic_user_count/mapper.py \
                        -mapper ~/git/Social/Social/src/hadoop/hot_topic_user_count/mapper.py \
                        -file ~/git/Social/Social/src/hadoop/hot_topic_user_count/reducer.py \
                        -reducer ~/git/Social/Social/src/hadoop/hot_topic_user_count/reducer.py \
                        -input /Social/input -output /Social/output
                    '''
        subprocess.call(run_hadoop, shell=True)
        #output
        get_file = '~/hadoop-2.6.0/bin/hdfs dfs -get /Social/output '\
                    '~/git/Social/Social/resource/hot_topic_user_count'
        subprocess.call(get_file, shell=True)
        
        sort_source = '/home/ivy/git/Social/Social/resource/hot_topic_user_count/output/part-00000'
        sort_target = '/home/ivy/git/Social/Social/resource/hot_topic_user_count/%d.txt' % date_num
        subprocess.Popen("sort -t '\t' -k1 -k3n -k4nr", shell=True,
                        stdin=open(sort_source, 'r'),
                        stdout=open(sort_target, 'w'),  # Not a pipe here,
                        )
#         head = subprocess.Popen(['head', '-100'],
#                         stdin=sort.stdout,
#                         stdout=open('/home/ivy/git/Social/Social/resource/wordcount/%d.txt' % date_num, 'w'),  # Not a pipe here
#                         )
    
if __name__ == '__main__':
    main() 